{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After downloading the selected database of ECG signals, a preprocessing stage was put in place to obtain an adequate organization of the data, extracting general information of each subject and specific characteristics of each ECG waveform to create a DataFrame with several features for further processing. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing packages\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import scipy as sp\n",
    "import scipy.signal\n",
    "import glob\n",
    "import os\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 Signal pre-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since there are relative few subjects, more data was obtained by dividing the signal into portions, assigning the correspondet information to create more rows with unique values. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting the ECG signals from original files\n",
    "\n",
    "general_directory = os.path.join(os.getcwd(), 'Data\\ecg-id-database-1.0.0\\ecg-id-database-1.0.0')\n",
    "entries = os.listdir(general_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extracting relevant information and creating basic ECG based features\n",
    "\n",
    "fs = 500 # Sampling frequency indicated by the publisher of the database\n",
    "subject, age, gender, RR, ecg_mean, ecg_std, ecg_var, ecg_median, ecg_samples = [], [], [], [], [], [], [], [], []\n",
    "\n",
    "for folder in entries:\n",
    "    if \"Person_\" in folder:\n",
    "             \n",
    "        info_targetPattern = os.path.join(general_directory, folder, '*.hea')\n",
    "        info_files = glob.glob(info_targetPattern)\n",
    "        \n",
    "        for ecg_info in info_files:\n",
    "            with open(ecg_info) as f:\n",
    "                subject_info = int(folder.replace(\"Person_\",\"\", 1))\n",
    "                \n",
    "                # Extracting demographic information\n",
    "                lines = [line.rstrip() for line in f]\n",
    "                age_info = int([int(s) for s in lines[4].split() if s.isdigit()][0])\n",
    "                condition = lines[5].find('female')\n",
    "                gender_info = 'female' if condition > 0 else 'male'\n",
    "                \n",
    "                ecg_signal_file = ecg_info.replace(\".hea\", \".dat\")\n",
    "                ecg_signal = np.fromfile(ecg_signal_file, dtype='int32')\n",
    "                ecg_signal = ecg_signal/max(ecg_signal)\n",
    "                ts = np.arange(0, len(ecg_signal)/fs, 1/fs)\n",
    "                \n",
    "                # RR intervals calculation\n",
    "                pks_RR = sp.signal.find_peaks(ecg_signal, height=np.mean(np.abs(ecg_signal)*5), distance=500)[0]\n",
    "                \n",
    "                RR_samples = np.diff(pks_RR)\n",
    "                RR_time = RR_samples*ts[1] \n",
    "                \n",
    "                # Filling row gaps with blank spaces\n",
    "                if len(pks_RR) <= 1:\n",
    "                    subject.append(subject_info)\n",
    "                    age.append(age_info)\n",
    "                    gender.append(gender_info)\n",
    "                    \n",
    "                    ecg_mean.append('')\n",
    "                    ecg_std.append('')\n",
    "                    ecg_var.append('')\n",
    "                    ecg_median.append('')\n",
    "                    RR.append('')\n",
    "                    ecg_samples.append('')\n",
    "                    \n",
    "                # Assigning values to the correspondent row\n",
    "                for index in range(len(pks_RR)-1):\n",
    "                    \n",
    "                    subject.append(subject_info)\n",
    "                    age.append(age_info)\n",
    "                    gender.append(gender_info)\n",
    "                    \n",
    "                    ecg_mean.append(np.mean(ecg_signal[pks_RR[index]:pks_RR[index+1]]))\n",
    "                    ecg_std.append(np.std(ecg_signal[pks_RR[index]:pks_RR[index+1]]))\n",
    "                    ecg_var.append(np.var(ecg_signal[pks_RR[index]:pks_RR[index+1]]))\n",
    "                    ecg_median.append(np.median(ecg_signal[pks_RR[index]:pks_RR[index+1]]))\n",
    "                    RR.append(RR_time[index])\n",
    "                    \n",
    "                    ecg_samples.append(ecg_signal[pks_RR[index]:pks_RR[index+1]]) # Selecting a unique portion of the raw ECG signal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 DataFrame declaration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject_ID</th>\n",
       "      <th>Age</th>\n",
       "      <th>Gender</th>\n",
       "      <th>RR</th>\n",
       "      <th>ECG_mean</th>\n",
       "      <th>ECG_std</th>\n",
       "      <th>ECG_var</th>\n",
       "      <th>ECG_median</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>male</td>\n",
       "      <td>1.564</td>\n",
       "      <td>-0.00570945</td>\n",
       "      <td>0.14325</td>\n",
       "      <td>0.0205207</td>\n",
       "      <td>-0.0440251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>male</td>\n",
       "      <td>1.864</td>\n",
       "      <td>0.00910373</td>\n",
       "      <td>0.148422</td>\n",
       "      <td>0.022029</td>\n",
       "      <td>-0.037736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>male</td>\n",
       "      <td>1.852</td>\n",
       "      <td>-0.00195572</td>\n",
       "      <td>0.15066</td>\n",
       "      <td>0.0226984</td>\n",
       "      <td>-0.0440252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>male</td>\n",
       "      <td>1.756</td>\n",
       "      <td>-0.00592427</td>\n",
       "      <td>0.156182</td>\n",
       "      <td>0.0243929</td>\n",
       "      <td>-0.050315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>male</td>\n",
       "      <td>2.496</td>\n",
       "      <td>0.0216589</td>\n",
       "      <td>0.150437</td>\n",
       "      <td>0.0226313</td>\n",
       "      <td>-0.0314467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2854</th>\n",
       "      <td>89</td>\n",
       "      <td>40</td>\n",
       "      <td>female</td>\n",
       "      <td>2.79</td>\n",
       "      <td>0.00345874</td>\n",
       "      <td>0.12851</td>\n",
       "      <td>0.0165149</td>\n",
       "      <td>-0.0397724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2855</th>\n",
       "      <td>89</td>\n",
       "      <td>40</td>\n",
       "      <td>female</td>\n",
       "      <td>1.712</td>\n",
       "      <td>0.00398307</td>\n",
       "      <td>0.133123</td>\n",
       "      <td>0.0177218</td>\n",
       "      <td>-0.0397721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2856</th>\n",
       "      <td>89</td>\n",
       "      <td>40</td>\n",
       "      <td>female</td>\n",
       "      <td>1.662</td>\n",
       "      <td>-0.000443603</td>\n",
       "      <td>0.130906</td>\n",
       "      <td>0.0171364</td>\n",
       "      <td>-0.0454548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857</th>\n",
       "      <td>90</td>\n",
       "      <td>21</td>\n",
       "      <td>female</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2858</th>\n",
       "      <td>90</td>\n",
       "      <td>21</td>\n",
       "      <td>female</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2859 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Subject_ID  Age  Gender     RR     ECG_mean   ECG_std    ECG_var  \\\n",
       "0              1   25    male  1.564  -0.00570945   0.14325  0.0205207   \n",
       "1              1   25    male  1.864   0.00910373  0.148422   0.022029   \n",
       "2              1   25    male  1.852  -0.00195572   0.15066  0.0226984   \n",
       "3              1   25    male  1.756  -0.00592427  0.156182  0.0243929   \n",
       "4              1   25    male  2.496    0.0216589  0.150437  0.0226313   \n",
       "...          ...  ...     ...    ...          ...       ...        ...   \n",
       "2854          89   40  female   2.79   0.00345874   0.12851  0.0165149   \n",
       "2855          89   40  female  1.712   0.00398307  0.133123  0.0177218   \n",
       "2856          89   40  female  1.662 -0.000443603  0.130906  0.0171364   \n",
       "2857          90   21  female                                            \n",
       "2858          90   21  female                                            \n",
       "\n",
       "     ECG_median  \n",
       "0    -0.0440251  \n",
       "1     -0.037736  \n",
       "2    -0.0440252  \n",
       "3     -0.050315  \n",
       "4    -0.0314467  \n",
       "...         ...  \n",
       "2854 -0.0397724  \n",
       "2855 -0.0397721  \n",
       "2856 -0.0454548  \n",
       "2857             \n",
       "2858             \n",
       "\n",
       "[2859 rows x 8 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# DataFrame declaration\n",
    "\n",
    "data={\"Subject_ID\": subject, \"Age\": np.array(age), \"Gender\": gender, \"RR\": RR, \"ECG_mean\": ecg_mean, \"ECG_std\": ecg_std,\n",
    "      \"ECG_var\": ecg_var, \"ECG_median\": ecg_median}\n",
    "df_ecg = pd.DataFrame(data)\n",
    "df_ecg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2859 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject_ID</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2859.000000</td>\n",
       "      <td>2859.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>38.874432</td>\n",
       "      <td>28.857643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>26.443411</td>\n",
       "      <td>11.864158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>13.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>13.000000</td>\n",
       "      <td>21.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>39.000000</td>\n",
       "      <td>23.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>61.000000</td>\n",
       "      <td>34.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>90.000000</td>\n",
       "      <td>75.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Subject_ID          Age\n",
       "count  2859.000000  2859.000000\n",
       "mean     38.874432    28.857643\n",
       "std      26.443411    11.864158\n",
       "min       1.000000    13.000000\n",
       "25%      13.000000    21.000000\n",
       "50%      39.000000    23.000000\n",
       "75%      61.000000    34.000000\n",
       "max      90.000000    75.000000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# DataFrame description\n",
    "\n",
    "num_rows = df_ecg.shape[0] #Provide the number of rows in the dataset\n",
    "num_cols = df_ecg.shape[1] #Provide the number of columns in the dataset\n",
    "\n",
    "print(num_rows, num_cols)\n",
    "df_ecg.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "According to column type, there are 6 categorical columns within the DataFrame. However, 5 of those columns should be numerical. To proceed, it has been identified that these columns present blank spaces in case of missing values. Thus, blank spaces are replaced with NaN, resulting in numerical columns. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Subject_ID     int64\n",
      "Age            int32\n",
      "Gender        object\n",
      "RR            object\n",
      "ECG_mean      object\n",
      "ECG_std       object\n",
      "ECG_var       object\n",
      "ECG_median    object\n",
      "dtype: object\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Gender        0\n",
       "RR            0\n",
       "ECG_mean      0\n",
       "ECG_std       0\n",
       "ECG_var       0\n",
       "ECG_median    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking categorical columns of the DataFrame\n",
    "\n",
    "print(df_ecg.dtypes)\n",
    "\n",
    "cat_df = df_ecg.select_dtypes(include=[\"object\"])\n",
    "cat_df.shape[1]\n",
    "cat_df.isnull().sum().sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Subject_ID      int64\n",
       "Age             int32\n",
       "Gender         object\n",
       "RR            float64\n",
       "ECG_mean      float64\n",
       "ECG_std       float64\n",
       "ECG_var       float64\n",
       "ECG_median    float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replacing blank spaces with NaN, resulting in numerical columns instead of categorical type\n",
    "\n",
    "df_ecg[\"RR\"] = df_ecg[\"RR\"].replace(\"\", np.nan)\n",
    "df_ecg[\"ECG_mean\"] = df_ecg[\"ECG_mean\"].replace(\"\", np.nan)\n",
    "df_ecg[\"ECG_std\"] = df_ecg[\"ECG_std\"].replace(\"\", np.nan)\n",
    "df_ecg[\"ECG_var\"] = df_ecg[\"ECG_var\"].replace(\"\", np.nan)\n",
    "df_ecg[\"ECG_median\"] = df_ecg[\"ECG_median\"].replace(\"\", np.nan)\n",
    "\n",
    "df_ecg.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Are the selected features suitable for classifying correctly the subjects based on ECG and basic demographic data?\n",
    "From the description of the DataFrame, the RR interval maximum value is too high, showing potential issues with the calculation method seleted. These outliers could introduce some bias and, thus, impact all the other features due to the way they are proposed. There are potential solutions to correct the implementation, such as a more robust algorithm to calculate the RR interval, fr example a Pan-Tompkins algorithm. However, it could be interesting to assess the impact of these empirical calculations to the trained model proposed for classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject_ID</th>\n",
       "      <th>Age</th>\n",
       "      <th>RR</th>\n",
       "      <th>ECG_mean</th>\n",
       "      <th>ECG_std</th>\n",
       "      <th>ECG_var</th>\n",
       "      <th>ECG_median</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2859.000000</td>\n",
       "      <td>2859.000000</td>\n",
       "      <td>2832.000000</td>\n",
       "      <td>2832.000000</td>\n",
       "      <td>2832.000000</td>\n",
       "      <td>2832.000000</td>\n",
       "      <td>2832.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>38.874432</td>\n",
       "      <td>28.857643</td>\n",
       "      <td>1.771206</td>\n",
       "      <td>0.003868</td>\n",
       "      <td>0.173795</td>\n",
       "      <td>0.031865</td>\n",
       "      <td>-0.043310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>26.443411</td>\n",
       "      <td>11.864158</td>\n",
       "      <td>0.816328</td>\n",
       "      <td>0.009374</td>\n",
       "      <td>0.040755</td>\n",
       "      <td>0.017201</td>\n",
       "      <td>0.019152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.131275</td>\n",
       "      <td>0.074087</td>\n",
       "      <td>0.005489</td>\n",
       "      <td>-0.153844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>13.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>1.380000</td>\n",
       "      <td>0.000496</td>\n",
       "      <td>0.147292</td>\n",
       "      <td>0.021695</td>\n",
       "      <td>-0.053695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>39.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>1.628000</td>\n",
       "      <td>0.003803</td>\n",
       "      <td>0.162338</td>\n",
       "      <td>0.026354</td>\n",
       "      <td>-0.043322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>61.000000</td>\n",
       "      <td>34.000000</td>\n",
       "      <td>1.958000</td>\n",
       "      <td>0.006981</td>\n",
       "      <td>0.196118</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>-0.032924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>90.000000</td>\n",
       "      <td>75.000000</td>\n",
       "      <td>13.722000</td>\n",
       "      <td>0.153532</td>\n",
       "      <td>0.567605</td>\n",
       "      <td>0.322175</td>\n",
       "      <td>0.095204</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Subject_ID          Age           RR     ECG_mean      ECG_std  \\\n",
       "count  2859.000000  2859.000000  2832.000000  2832.000000  2832.000000   \n",
       "mean     38.874432    28.857643     1.771206     0.003868     0.173795   \n",
       "std      26.443411    11.864158     0.816328     0.009374     0.040755   \n",
       "min       1.000000    13.000000     1.000000    -0.131275     0.074087   \n",
       "25%      13.000000    21.000000     1.380000     0.000496     0.147292   \n",
       "50%      39.000000    23.000000     1.628000     0.003803     0.162338   \n",
       "75%      61.000000    34.000000     1.958000     0.006981     0.196118   \n",
       "max      90.000000    75.000000    13.722000     0.153532     0.567605   \n",
       "\n",
       "           ECG_var   ECG_median  \n",
       "count  2832.000000  2832.000000  \n",
       "mean      0.031865    -0.043310  \n",
       "std       0.017201     0.019152  \n",
       "min       0.005489    -0.153844  \n",
       "25%       0.021695    -0.053695  \n",
       "50%       0.026354    -0.043322  \n",
       "75%       0.038462    -0.032924  \n",
       "max       0.322175     0.095204  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ecg.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Gender', 'Subject_ID', 'Age'}\n",
      "set()\n"
     ]
    }
   ],
   "source": [
    "no_nulls = set(df_ecg.dropna(axis=\"columns\").columns) # Provide a set of columns with 0 missing values.\n",
    "print(no_nulls)\n",
    "most_missing_cols = set(df_ecg.columns[df_ecg.isnull().sum() > len(df_ecg)*0.05]) # Provide a set of columns with more than 5% of the values missing\n",
    "print(most_missing_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ecg.to_pickle(\"df_ecg\")\n",
    "\n",
    "df_ecg_samples = pd.DataFrame(ecg_samples).add_prefix('ecg_')\n",
    "df_ecg_samples.to_pickle(\"df_ecg_samples\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.3 Conclusions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The pre-processing of the ECG signal database allows to construct a DataFrame with selected information regarding basic demographic data and calculated ECG features. \n",
    "Additionally, a portion of the raw (original) ECG signal was also saved for further analysis.\n",
    "Feature engineering could entail a more complex and robust processing of the data but the proposed features could demonstrate if the trained model could have a good performance even with calculation pitfalls or the raw data."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
